library(mlr3)
library(mlr3learners)
library(mlr3tuning)
library(paradox)

data(Pima.te, package = "MASS")
data(Pima.tr, package = "MASS")
pima <- rbind(Pima.te, Pima.tr)

pima.scale <- data.frame(scale(pima[, -8]))
pima.scale$type <- pima$type

pima.task <- TaskClassif$new(id = "pima", backend = pima.scale, target = "type")
pima.svm.linear <- lrn("classif.svm", id = "svm.linear",
                       predict_type = "prob",
                       type = "C-classification", kernel = "linear")


pp <- as.data.table(pima.svm$param_set)

# Tuning augments
resampling <- rsmp("holdout")
measure <- msr("classif.ce")
terminator <- term("evals", n_evals = 10)
tuner <- tnr("random_search")

# Linear kernel ----
tune.ps.linear <- ParamSet$new(list(
  ParamDbl$new("cost", lower = 0.001, upper = 1)
))

at.linear <- AutoTuner$new(
  learner = pima.svm.linear,
  resampling = resampling,
  measures = measure,
  tune_ps = tune.ps.linear,
  terminator = terminator,
  tuner = tuner
)

at.linear$param_set$values

# Learner is NULL before training.
at.linear$train(pima.task)  # at$train() will tune and train.

# AutoTuner wrap a learner that is not identical to the autotuner.
at.linear$learner$param_set$values

pred.linear <- at.linear$predict(pima.task)
pred.linear$score()

# Polynomial kernel ----
pima.svm.poly <- lrn("classif.svm", id = "svm.poly", 
                     predict_type = "prob",
                     type = "C-classification", kernel = "polynomial")

tune.ps.poly <- ParamSet$new(list(
  ParamInt$new("degree", lower = 3, upper = 5),
  ParamDbl$new("coef0", lower = 0.1, upper = 4)
))

at.poly <- AutoTuner$new(
  learner = pima.svm.poly,
  resampling = resampling,
  measures = measure,
  tune_ps = tune.ps.poly,
  terminator = terminator,
  tuner = tuner
)

at.poly$train(pima.task)
at.poly$learner$param_set$values

pred.poly <- at.poly$predict(pima.task)
pred.poly$score()

# Radial kernel ----
pima.svm.rad <- lrn("classif.svm", id = "svm.radial",
                    predict_type = "prob",
                    type = "C-classification", kernel = "radial")

tune.ps.rad <- ParamSet$new(list(
  ParamDbl$new("gamma", lower = 0.1, upper = 4)
))

at.rad <- AutoTuner$new(
  learner = pima.svm.rad,
  resampling = resampling,
  measures = measure,
  tune_ps = tune.ps.rad,
  terminator = terminator,
  tuner = tuner
)

at.rad$train(pima.task)
at.poly$learner$param_set$values
at.poly$param_set$values

pred.rad <- at.poly$predict(pima.task)
pred.rad$score()

# Benchmarking ----
bmd <- benchmark_grid(
  tasks = pima.task,
  learners = list(at.linear, at.poly, at.rad),
  resamplings = rsmp("cv", folds = 3)
)
bmr <- benchmark(bmd)

bmr$aggregate(measure)

library(mlr3viz)
# autoplot(bmr)
autoplot(bmr, type = "roc")
